from collections import Counter
from config import conf
import pandas as pd


def do():
    eda()
    # gen_labels_dict()


def eda_one(path):
    data = pd.read_csv(path, sep='\t', names=['label', 'text'])
    print(f'数据查看\n{data.head()}')
    print(f'数据概览')
    data.info()
    print(f'数据标签分布')
    print(Counter(data['label']))
    data['len'] = data['text'].apply(lambda x: len(x))
    print(f'数据长度分布')
    print(data['len'].describe())


def gen_labels_dict():
    data = pd.read_csv(conf.test_path, sep='\t', names=['label', 'text'])
    labels = []
    for label in Counter(data['label']):
        labels.append(label)

    id2label = {i: label for i, label in enumerate(labels)}
    label2id = {label: i for i, label in enumerate(labels)}
    with open(conf.id2label_path, 'w', encoding='utf-8') as f:
        f.write(str(id2label))
    with open(conf.label2id_path, 'w', encoding='utf-8') as f:
        f.write(str(label2id))


def eda():
    print('分析训练数据')
    eda_one(conf.train_path)
    print('分析测试数据')
    eda_one(conf.test_path)
    print('分析验证数据')
    eda_one(conf.val_path)
    """
    数据量：训练5w,测试1w,验证5k
    标签均衡
    训练集句子长度：mean=913，std=930，==》3703
    """


if __name__ == '__main__':
    do()
